unit QTable;

interface

type

  TAction = (acUp, acRight, acDown, acLeft);
  TStateType = (stInitial, stClean, stWall, stHole, stMonster, stGold, stWind, stSmell);
  
  TState = class
  private
    FRow, FCol : Integer;
    FReward   : Double;
    FStateType     : TStateType;
    procedure SetState (Row, Col : Integer);
  public
    constructor Create (Row, Col : Integer);

    property Row : Integer read FRow;
    property Col : Integer read FCol;
    property Reward : Double read FReward write FReward;
    property StateType : TStateType read FStateType write FStateType;
   end;

  TQTable = class
  private
    FRows, FCols : Integer;
    FQValue : array of array of array of Double;
    function GetQValue(State : TState; Action : TAction): Double;
    procedure SetQValue(State: TState; Action : TAction; const Value: Double);

  public
    constructor Create (Rows, Cols, Actions : Integer);

    function BestQ (State : TState) : Double;
    function BestAction (State : TState) : TAction;

    property QValue[State : TState; Action : TAction] : Double read GetQValue write SetQValue;
    property Rows : Integer read FRows;
    property Cols : Integer read FCols;
    
  end;

implementation

uses SysUtils;

{ TQTable }

function TQTable.BestAction(State: TState): TAction;
var
  Ix : TAction;
  Best, Value : Double;

  BestActions : set of TAction;  
  
begin
  Best := -999999999;
  Result := Low (TAction);
  BestActions := [];
  for Ix := Low (TAction) to High (TAction) do
    begin
      if (QValue[State, Ix] > Best) then
        begin
          Best := QValue[State, Ix];
          BestActions := [Ix];
        end
      else if (QValue[State, Ix] = Best) then
        begin
          BestActions := BestActions + [Ix];
        end;
    end;

  Best := 0;
  for Ix := Low (TAction) to High (TAction) do
    begin
      if Ix in BestActions then
        begin
          Value := Random;
          if Value >= Best then
            begin
              Best := Value;
              Result := Ix;
            end;
        end;
    end;
end;

function TQTable.BestQ(State: TState): Double;
var
  Ix : TAction;
  Best : Double;
begin
  Best := -999999999;
  for Ix := Low (TAction) to High (TAction) do
    begin
      if (QValue[State, Ix] > Best) then
        begin
          Best := QValue[State, Ix];
        end;
    end;

  Result := Best;
end;

constructor TQTable.Create (Rows, Cols, Actions : Integer);
var
  Ix, Jx : Integer;
  Acx     : TAction;

  State : TState;
const
  INITAL_VALUE = 0;

begin
  FRows := Rows;
  FCols := Cols;
  SetLength (FQValue, Rows, Cols, Integer (High (TAction)) + 1);

  for Ix := 0 to Rows-1 do
    begin
      for Jx := 0 to Cols-1 do
        begin
          State := TState.Create(Ix, Jx);

          for Acx := Low (TAction) to High (TAction) do
            begin
              QValue[State, Acx]     := INITAL_VALUE;
            end;

          FreeAndNil(State);
        end;
    end;

end;

function TQTable.GetQValue(State : TState; Action : TAction): Double;
begin
  Result := FQValue[State.Row, State.Col, Integer(Action)];
end;

procedure TQTable.SetQValue(State: TState; Action : TAction;
  const Value: Double);
begin
  FQValue[State.Row, State.Col, Integer(Action)] := Value;
end;

{ TState }

constructor TState.Create(Row, Col: Integer);
begin
  SetState(Row, Col);
end;

procedure TState.SetState(Row, Col: Integer);
begin
  FRow := Row;
  FCol := Col;
end;

end.
